{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 协同过滤算法特征工程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle as pk\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "#字典，用于建立用户和物品的索引\n",
    "from collections import defaultdict\n",
    "#稀疏矩阵，存储打分表\n",
    "import scipy.io as sio\n",
    "import scipy.sparse as ss\n",
    "import copy\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = './Data/'\n",
    "model_path = './model/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>msno</th>\n",
       "      <th>song_id</th>\n",
       "      <th>source_system_tab</th>\n",
       "      <th>source_screen_name</th>\n",
       "      <th>source_type</th>\n",
       "      <th>target</th>\n",
       "      <th>bd</th>\n",
       "      <th>registration_init_time</th>\n",
       "      <th>expiration_date</th>\n",
       "      <th>song_length</th>\n",
       "      <th>genre_ids</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>composer</th>\n",
       "      <th>lyricist</th>\n",
       "      <th>name</th>\n",
       "      <th>missing_count</th>\n",
       "      <th>song_length_section</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>201969</th>\n",
       "      <td>FGtllVqz18RPiwJj/edr2gV78zirAiY/9SmYvia+kCg=</td>\n",
       "      <td>BBzumQNXUHKdEBOB7mAJuzok+IJA1c2Ryg/yzTF6tik=</td>\n",
       "      <td>explore</td>\n",
       "      <td>Explore</td>\n",
       "      <td>online-playlist</td>\n",
       "      <td>1</td>\n",
       "      <td>28.734196</td>\n",
       "      <td>20120102</td>\n",
       "      <td>20171005</td>\n",
       "      <td>206471.0</td>\n",
       "      <td>359</td>\n",
       "      <td>Bastille</td>\n",
       "      <td>Dan Smith| Mark Crew</td>\n",
       "      <td>0</td>\n",
       "      <td>Good Grief</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1932462</th>\n",
       "      <td>Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=</td>\n",
       "      <td>bhp/MpSNoqoxOIB+/l8WPqu6jldth4DIpCm3ayXnJqM=</td>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-playlist</td>\n",
       "      <td>1</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>20110525</td>\n",
       "      <td>20170911</td>\n",
       "      <td>284584.0</td>\n",
       "      <td>1259</td>\n",
       "      <td>Various Artists</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Lords of Cardboard</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183559</th>\n",
       "      <td>Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=</td>\n",
       "      <td>JNWfrrC7zNN7BdMpsISKa4Mw+xVJYNnxXh3/Epw7QgY=</td>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-playlist</td>\n",
       "      <td>1</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>20110525</td>\n",
       "      <td>20170911</td>\n",
       "      <td>225396.0</td>\n",
       "      <td>1259</td>\n",
       "      <td>Nas</td>\n",
       "      <td>N. Jones、W. Adams、J. Lordan、D. Ingle</td>\n",
       "      <td>0</td>\n",
       "      <td>Hip Hop Is Dead(Album Version (Edited))</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149511</th>\n",
       "      <td>Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=</td>\n",
       "      <td>2A87tzfnJTSWqD7gIZHisolhe4DMdzkbd6LzO1KHjNs=</td>\n",
       "      <td>my library</td>\n",
       "      <td>Local playlist more</td>\n",
       "      <td>local-playlist</td>\n",
       "      <td>1</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>20110525</td>\n",
       "      <td>20170911</td>\n",
       "      <td>255512.0</td>\n",
       "      <td>1019</td>\n",
       "      <td>Soundway</td>\n",
       "      <td>Kwadwo Donkoh</td>\n",
       "      <td>0</td>\n",
       "      <td>Disco Africa</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74867</th>\n",
       "      <td>FGtllVqz18RPiwJj/edr2gV78zirAiY/9SmYvia+kCg=</td>\n",
       "      <td>3qm6XTZ6MOCU11x8FIVbAGH5l5uMkT3/ZalWG1oo2Gc=</td>\n",
       "      <td>explore</td>\n",
       "      <td>Explore</td>\n",
       "      <td>online-playlist</td>\n",
       "      <td>1</td>\n",
       "      <td>28.734196</td>\n",
       "      <td>20120102</td>\n",
       "      <td>20171005</td>\n",
       "      <td>187802.0</td>\n",
       "      <td>1011</td>\n",
       "      <td>Brett Young</td>\n",
       "      <td>Brett Young| Kelly Archer| Justin Ebach</td>\n",
       "      <td>0</td>\n",
       "      <td>Sleep Without You</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 msno  \\\n",
       "201969   FGtllVqz18RPiwJj/edr2gV78zirAiY/9SmYvia+kCg=   \n",
       "1932462  Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=   \n",
       "183559   Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=   \n",
       "149511   Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=   \n",
       "74867    FGtllVqz18RPiwJj/edr2gV78zirAiY/9SmYvia+kCg=   \n",
       "\n",
       "                                              song_id source_system_tab  \\\n",
       "201969   BBzumQNXUHKdEBOB7mAJuzok+IJA1c2Ryg/yzTF6tik=           explore   \n",
       "1932462  bhp/MpSNoqoxOIB+/l8WPqu6jldth4DIpCm3ayXnJqM=        my library   \n",
       "183559   JNWfrrC7zNN7BdMpsISKa4Mw+xVJYNnxXh3/Epw7QgY=        my library   \n",
       "149511   2A87tzfnJTSWqD7gIZHisolhe4DMdzkbd6LzO1KHjNs=        my library   \n",
       "74867    3qm6XTZ6MOCU11x8FIVbAGH5l5uMkT3/ZalWG1oo2Gc=           explore   \n",
       "\n",
       "          source_screen_name      source_type  target         bd  \\\n",
       "201969               Explore  online-playlist       1  28.734196   \n",
       "1932462  Local playlist more   local-playlist       1  24.000000   \n",
       "183559   Local playlist more   local-playlist       1  24.000000   \n",
       "149511   Local playlist more   local-playlist       1  24.000000   \n",
       "74867                Explore  online-playlist       1  28.734196   \n",
       "\n",
       "         registration_init_time  expiration_date  song_length genre_ids  \\\n",
       "201969                 20120102         20171005     206471.0       359   \n",
       "1932462                20110525         20170911     284584.0      1259   \n",
       "183559                 20110525         20170911     225396.0      1259   \n",
       "149511                 20110525         20170911     255512.0      1019   \n",
       "74867                  20120102         20171005     187802.0      1011   \n",
       "\n",
       "             artist_name                                 composer lyricist  \\\n",
       "201969          Bastille                     Dan Smith| Mark Crew        0   \n",
       "1932462  Various Artists                                        0        0   \n",
       "183559               Nas     N. Jones、W. Adams、J. Lordan、D. Ingle        0   \n",
       "149511          Soundway                            Kwadwo Donkoh        0   \n",
       "74867        Brett Young  Brett Young| Kelly Archer| Justin Ebach        0   \n",
       "\n",
       "                                            name  missing_count  \\\n",
       "201969                                Good Grief              3   \n",
       "1932462                       Lords of Cardboard              2   \n",
       "183559   Hip Hop Is Dead(Album Version (Edited))              1   \n",
       "149511                              Disco Africa              1   \n",
       "74867                          Sleep Without You              3   \n",
       "\n",
       "         song_length_section  \n",
       "201969                     1  \n",
       "1932462                    1  \n",
       "183559                     1  \n",
       "149511                     1  \n",
       "74867                      1  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取 EDA 分析的结果\n",
    "with open(model_path + 'data_train_after_drop_city_v1.pkl','rb') as fr:\n",
    "    data_train_v1 = pk.load(fr)\n",
    "fr.close()\n",
    "data_train_v1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 提取 msno song_id target , 建立打分矩阵的原始数据\n",
    "data_train_target_origin = data_train_v1.loc[:, ['msno', 'song_id', 'target']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>msno</th>\n",
       "      <th>song_id</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>201969</th>\n",
       "      <td>FGtllVqz18RPiwJj/edr2gV78zirAiY/9SmYvia+kCg=</td>\n",
       "      <td>BBzumQNXUHKdEBOB7mAJuzok+IJA1c2Ryg/yzTF6tik=</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1932462</th>\n",
       "      <td>Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=</td>\n",
       "      <td>bhp/MpSNoqoxOIB+/l8WPqu6jldth4DIpCm3ayXnJqM=</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183559</th>\n",
       "      <td>Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=</td>\n",
       "      <td>JNWfrrC7zNN7BdMpsISKa4Mw+xVJYNnxXh3/Epw7QgY=</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149511</th>\n",
       "      <td>Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=</td>\n",
       "      <td>2A87tzfnJTSWqD7gIZHisolhe4DMdzkbd6LzO1KHjNs=</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74867</th>\n",
       "      <td>FGtllVqz18RPiwJj/edr2gV78zirAiY/9SmYvia+kCg=</td>\n",
       "      <td>3qm6XTZ6MOCU11x8FIVbAGH5l5uMkT3/ZalWG1oo2Gc=</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 msno  \\\n",
       "201969   FGtllVqz18RPiwJj/edr2gV78zirAiY/9SmYvia+kCg=   \n",
       "1932462  Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=   \n",
       "183559   Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=   \n",
       "149511   Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=   \n",
       "74867    FGtllVqz18RPiwJj/edr2gV78zirAiY/9SmYvia+kCg=   \n",
       "\n",
       "                                              song_id  target  \n",
       "201969   BBzumQNXUHKdEBOB7mAJuzok+IJA1c2Ryg/yzTF6tik=       1  \n",
       "1932462  bhp/MpSNoqoxOIB+/l8WPqu6jldth4DIpCm3ayXnJqM=       1  \n",
       "183559   JNWfrrC7zNN7BdMpsISKa4Mw+xVJYNnxXh3/Epw7QgY=       1  \n",
       "149511   2A87tzfnJTSWqD7gIZHisolhe4DMdzkbd6LzO1KHjNs=       1  \n",
       "74867    3qm6XTZ6MOCU11x8FIVbAGH5l5uMkT3/ZalWG1oo2Gc=       1  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_train_target_origin.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7377418, 3)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_train_target_origin.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 用户对哪些 item 打过分\n",
    "train_user_item_score_series = data_train_target_origin['target'].groupby([data_train_target_origin['msno'], data_train_target_origin['song_id']]).sum()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "msno                                          song_id                                     \n",
       "++5wYjoMgQHoRuD3GbbvmphZbBBwymzv5Q4l8sywtuU=  +/lcxtBy9FuH0ObLsK9wRf3zl9zSyvDNMpTWSGCAXxc=    0\n",
       "                                              +JGuj3rm4FBs8loN7rvI+JZ+EX3K9+WaxbDtmjs6mQc=    1\n",
       "                                              +MRnGH0Gg7jA7izLFRU1SZtGPmWHdsWTeL9wRXChnRA=    1\n",
       "                                              +Sm75wnBf/sjm/QMUAFx8N+Ae04kWCXGlgH50tTeM6c=    1\n",
       "                                              +d62ngXhdNTJRLKXO8/X9+BBoj77Hs8xVHMLmYGmB4k=    0\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_user_item_score_series.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    3714656\n",
       "0    3662762\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_user_item_score_series.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp1 = train_user_item_score_series.reset_index()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "行数没有减少，说明没有用户对同一个歌曲重复打分。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n"
     ]
    }
   ],
   "source": [
    "print(type(temp1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 去除参与打分少的用户"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "msno_value_counts = temp1['msno'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp1['msno_judge'] = temp1['msno'].apply(lambda msno: 1 if msno_value_counts[msno] >5 else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    7369722\n",
       "0       7696\n",
       "Name: msno_judge, dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp1.msno_judge.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27764\n"
     ]
    }
   ],
   "source": [
    "temp2 = copy.deepcopy(temp1)\n",
    "temp2 = temp2[temp2.msno_judge == 1]\n",
    "temp2 = temp2.drop(['msno_judge'], axis = 1)\n",
    "print(len(temp2.msno.unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>msno</th>\n",
       "      <th>song_id</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>++5wYjoMgQHoRuD3GbbvmphZbBBwymzv5Q4l8sywtuU=</td>\n",
       "      <td>+/lcxtBy9FuH0ObLsK9wRf3zl9zSyvDNMpTWSGCAXxc=</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>++5wYjoMgQHoRuD3GbbvmphZbBBwymzv5Q4l8sywtuU=</td>\n",
       "      <td>+JGuj3rm4FBs8loN7rvI+JZ+EX3K9+WaxbDtmjs6mQc=</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>++5wYjoMgQHoRuD3GbbvmphZbBBwymzv5Q4l8sywtuU=</td>\n",
       "      <td>+MRnGH0Gg7jA7izLFRU1SZtGPmWHdsWTeL9wRXChnRA=</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>++5wYjoMgQHoRuD3GbbvmphZbBBwymzv5Q4l8sywtuU=</td>\n",
       "      <td>+Sm75wnBf/sjm/QMUAFx8N+Ae04kWCXGlgH50tTeM6c=</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>++5wYjoMgQHoRuD3GbbvmphZbBBwymzv5Q4l8sywtuU=</td>\n",
       "      <td>+d62ngXhdNTJRLKXO8/X9+BBoj77Hs8xVHMLmYGmB4k=</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           msno  \\\n",
       "0  ++5wYjoMgQHoRuD3GbbvmphZbBBwymzv5Q4l8sywtuU=   \n",
       "1  ++5wYjoMgQHoRuD3GbbvmphZbBBwymzv5Q4l8sywtuU=   \n",
       "2  ++5wYjoMgQHoRuD3GbbvmphZbBBwymzv5Q4l8sywtuU=   \n",
       "3  ++5wYjoMgQHoRuD3GbbvmphZbBBwymzv5Q4l8sywtuU=   \n",
       "4  ++5wYjoMgQHoRuD3GbbvmphZbBBwymzv5Q4l8sywtuU=   \n",
       "\n",
       "                                        song_id  target  \n",
       "0  +/lcxtBy9FuH0ObLsK9wRf3zl9zSyvDNMpTWSGCAXxc=       0  \n",
       "1  +JGuj3rm4FBs8loN7rvI+JZ+EX3K9+WaxbDtmjs6mQc=       1  \n",
       "2  +MRnGH0Gg7jA7izLFRU1SZtGPmWHdsWTeL9wRXChnRA=       1  \n",
       "3  +Sm75wnBf/sjm/QMUAFx8N+Ae04kWCXGlgH50tTeM6c=       1  \n",
       "4  +d62ngXhdNTJRLKXO8/X9+BBoj77Hs8xVHMLmYGmB4k=       0  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp2.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.series.Series'>\n"
     ]
    }
   ],
   "source": [
    "# 歌曲被哪些用户打过分\n",
    "train_item_user_score_series = temp2['target'].groupby([temp2['song_id'], temp2['msno']]).sum()\n",
    "print(type(train_item_user_score_series))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "song_id                                       msno                                        \n",
       "+++2AEoM0d8iZTdbnAjUm35bnGKGMXdZJSv4rrWK6JQ=  UK7HQJngD71i6BasobiKBbhBylFXWP4Kj6K7dVV4B0w=    0\n",
       "++/ACCkEN/+VtgrJxEqeRgRmV4y8pcarDJ9T/yRAi1E=  +5Z+tu8NySwd9J0mACPZTk/lcPYB1WgUp8ogKwocFXw=    0\n",
       "                                              jCkFTmE17pwgG5sjl7wSB+ah+RXYtYRMIlPUalskk1s=    0\n",
       "++/lJNswCU+za2pYB0cWIbGL5UzWIKtfweX20+GImZA=  LjuMIC48XR0RbjYU3YsUXqSINC1oSnyBL6rabbL+yRs=    0\n",
       "                                              NwX1NaVlyw0jvfQNxLTcx9ztIUKVL/sjkccdAmMpfCA=    0\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_item_user_score_series.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7369722, 3)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_users = temp2.msno.unique()\n",
    "unique_items = temp2.song_id.unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "users num: 27764\n",
      "items num: 359810\n"
     ]
    }
   ],
   "source": [
    "users_num = len(unique_users)\n",
    "items_num = len(unique_items)\n",
    "print(\"users num:\",len(unique_users))\n",
    "print(\"items num:\",len(unique_items))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 建立 用户和物品 的索引表\n",
    "users_index = dict()\n",
    "items_index = dict()\n",
    "for m,u in enumerate(unique_users):\n",
    "    users_index[u] = m\n",
    "for n,i in enumerate(unique_items):\n",
    "    items_index[i] = n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['++5wYjoMgQHoRuD3GbbvmphZbBBwymzv5Q4l8sywtuU=',\n",
       "       '++AH7m/EQ4iKe6wSlfO/xXAJx50p+fCeTyF90GoE9Pg=',\n",
       "       '++e+jsxuQ8UEnmW40od9Rq3rW7+wAum4wooXyZTKJpk=', ...,\n",
       "       'zzZBJUYXrb168A4Ff4sA8L2iOH0x4ciKdD6WkV53XaE=',\n",
       "       'zzompfSaMamqvjyCMYvgUBwYrxh8fHE40z3f73CQoak=',\n",
       "       'zzqc2ja7z10FtSpagYVcAZXg/gPRq7wcDZuNFj+zJSU='], dtype=object)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unique_users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Int64Index([      0,       1,       2,       3,       4,       5,       6,\n",
       "                  7,       8,       9,\n",
       "            ...\n",
       "            7377405, 7377406, 7377407, 7377408, 7377409, 7377410, 7377411,\n",
       "            7377412, 7377413, 7377414],\n",
       "           dtype='int64', length=7369722)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp2.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "msno       zzqc2ja7z10FtSpagYVcAZXg/gPRq7wcDZuNFj+zJSU=\n",
       "song_id    zqDZjACUVfphX2Me6LEbMwDWLXA4bIWCbSSD+QsIypQ=\n",
       "target                                                1\n",
       "Name: 7377414, dtype: object"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp2.loc[7377414,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "msno       ++5wYjoMgQHoRuD3GbbvmphZbBBwymzv5Q4l8sywtuU=\n",
       "song_id    22wN6a2Q/qVQosA8OBfVg6AbwvI2KJnZ35DXLqpLHJg=\n",
       "target                                                1\n",
       "Name: 29, dtype: object"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp2.iloc[29,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "msno       ++5wYjoMgQHoRuD3GbbvmphZbBBwymzv5Q4l8sywtuU=\n",
       "song_id    9YcQDUUollFlzS1DT8MqB+COuY3NAxU2aI5pKNcfVzk=\n",
       "target                                                1\n",
       "Name: 97, dtype: object"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp2.iloc[97,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 保存训练数据的索引表\n",
    "with open(model_path +'train_users_index.pkl', 'wb') as fw:\n",
    "    pk.dump(users_index, fw)\n",
    "fw.close()\n",
    "with open(model_path + 'train_items_index.pkl','wb') as fw:\n",
    "    pk.dump(items_index, fw)\n",
    "fw.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Int64Index([      0,       1,       2,       3,       4,       5,       6,\n",
       "                  7,       8,       9,\n",
       "            ...\n",
       "            7377405, 7377406, 7377407, 7377408, 7377409, 7377410, 7377411,\n",
       "            7377412, 7377413, 7377414],\n",
       "           dtype='int64', length=7369722)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp2.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "users_num: 27764 , items_num: 359810 \n"
     ]
    }
   ],
   "source": [
    "# 建立倒排表\n",
    "# 统计每个用户打过分的歌曲 / 每个歌曲被哪个用户打过分\n",
    "user_items = defaultdict(set)\n",
    "item_users = defaultdict(set)\n",
    "\n",
    "#用户-物品关系矩阵R, 稀疏矩阵，记录用户对每个电影的打分\n",
    "user_item_scores = ss.dok_matrix((users_num, items_num))\n",
    "print('users_num: %d , items_num: %d ' % (users_num, items_num))\n",
    "for line in temp2.index:\n",
    "    cur_user_index = users_index[temp2.loc[line].msno]\n",
    "    cur_item_index = items_index[temp2.loc[line].song_id]\n",
    "    user_items[cur_user_index].add(cur_item_index)\n",
    "    item_users[cur_item_index].add(cur_user_index)\n",
    "    \n",
    "    user_item_scores[cur_user_index, cur_item_index] = temp2.loc[line].target +1\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 保存 倒排表\n",
    "with open(model_path+'user_items.pkl', 'wb') as fw:\n",
    "    pk.dump(user_items,fw)\n",
    "fw.close()\n",
    "with open(model_path+'item_users.pkl', 'wb') as fw:\n",
    "    pk.dump(item_users,fw)\n",
    "fw.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 保存 user_item 的打分矩阵，在UserCF和ItemCF中用到\n",
    "sio.mmwrite(model_path+'user_item_scores.mtx', user_item_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 0)\t1.0\n",
      "  (0, 1)\t2.0\n",
      "  (0, 2)\t2.0\n",
      "  (0, 3)\t2.0\n",
      "  (0, 4)\t1.0\n",
      "  (0, 5)\t1.0\n",
      "  (0, 6)\t2.0\n",
      "  (0, 7)\t1.0\n",
      "  (0, 8)\t2.0\n",
      "  (0, 9)\t1.0\n",
      "  (0, 10)\t1.0\n",
      "  (0, 11)\t1.0\n",
      "  (0, 12)\t2.0\n",
      "  (0, 13)\t2.0\n",
      "  (0, 14)\t2.0\n",
      "  (0, 15)\t1.0\n",
      "  (0, 16)\t2.0\n",
      "  (0, 17)\t2.0\n",
      "  (0, 18)\t1.0\n",
      "  (0, 19)\t1.0\n",
      "  (0, 20)\t1.0\n",
      "  (0, 21)\t2.0\n",
      "  (0, 22)\t2.0\n",
      "  (0, 23)\t1.0\n",
      "  (0, 24)\t2.0\n",
      "  :\t:\n",
      "  (27763, 781)\t1.0\n",
      "  (27763, 129519)\t1.0\n",
      "  (27763, 7930)\t1.0\n",
      "  (27763, 13479)\t1.0\n",
      "  (27763, 1062)\t1.0\n",
      "  (27763, 37282)\t1.0\n",
      "  (27763, 6589)\t2.0\n",
      "  (27763, 54836)\t2.0\n",
      "  (27763, 782)\t1.0\n",
      "  (27763, 29193)\t2.0\n",
      "  (27763, 328790)\t1.0\n",
      "  (27763, 546)\t1.0\n",
      "  (27763, 2861)\t2.0\n",
      "  (27763, 28123)\t2.0\n",
      "  (27763, 1304)\t2.0\n",
      "  (27763, 4025)\t1.0\n",
      "  (27763, 40328)\t1.0\n",
      "  (27763, 231287)\t1.0\n",
      "  (27763, 6651)\t1.0\n",
      "  (27763, 2013)\t1.0\n",
      "  (27763, 13736)\t1.0\n",
      "  (27763, 3251)\t1.0\n",
      "  (27763, 111783)\t1.0\n",
      "  (27763, 6923)\t1.0\n",
      "  (27763, 226144)\t2.0\n"
     ]
    }
   ],
   "source": [
    "print(user_item_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(model_path+'temp2.pkl', 'wb') as fw:\n",
    "    pk.dump(temp2,fw)\n",
    "fw.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
